import os
import json

import requests
import MySQLdb

# import aliyunProvince
import pdb
import random
import time
def insert_geeks(cursor, js, check = False):
    for geek_info in js['data']['geekList']:
        try:
            company_js = json.loads(geek_info['company'])
        except Exception as e:
            company_js = {}
        company = company_js.get('company','')
        position = company_js.get('position','')
        introduce = geek_info.get('introduce','')
        name = geek_info['name'].replace('\'','\\\'')
        geek_id = str(geek_info['id'])
        projectExperiences = geek_info.get('projectExperiences','') # not parsed json data
        projectExperiences = projectExperiences.replace('\'','\\\'')
        if(len(projectExperiences) >= 1200):
            projectExperiences = ''
        if(check):
            sql_select_statement = 'select id from geek where id = \'%s\'' % geek_id
            cursor.execute(sql_select_statement)
            result = cursor.fetchone()
            if(result is not None):
                continue
        sql_statement = 'insert into geek (id, introduce, name, company, position, projectExperiences) values (\'{0}\',\'{1}\',\'{2}\',\'{3}\',\'{4}\',\'{5}\')'.format(
            geek_id, introduce, name, company, position, projectExperiences
        )
        try:
            cursor.execute(sql_statement)
        except Exception as e:
            print(e)
            pdb.set_trace()
           
if __name__ == '__main__':
    db = MySQLdb.connect(host="localhost", user="volunteer_geek", passwd="volunteer_geek", port=3306, db='volunteer_geek', charset='utf8mb4')
    cursor = db.cursor()
    TOTAL_GEEK = 4363
    GEEK_STYLE_URL = 'https://greencode.aliyun.com/geek/geekStyle.htm?spm=5176.100102.1007916.9.b5b639adHGI7a3'
    BASE_URL = 'https://greencode.aliyun.com/ajax/geekAjax/geekDemeanor.json'
    payload = {'skillId':'','keyWord':'','currentPage':1,'pageSize':12}
    request_times = int(TOTAL_GEEK / payload['pageSize'])+1
    start_page = 1
    if(os.path.exists('cache/cache.txt')):
        file_str = open('cache/cache.txt', 'rb').read().decode('utf-8')
        js = json.loads(file_str)
        start_page = js['data']['pageInfo']['currentPage'] + 1
        if(js['code']=='0'):
            insert_geeks(cursor, js, check = True)
            db.commit()
    
    s = requests.Session()
    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'}
    r = s.get(GEEK_STYLE_URL, headers=headers)
    headers['referer'] = GEEK_STYLE_URL
    for i in range(start_page, request_times + 1):
        payload['currentPage'] = i
        r = s.get(BASE_URL, params=payload, headers=headers)
        js = json.loads(r.text)
        if(js['code']!='0'):
            pdb.set_trace()
        open('cache/cache.txt','wb').write(r.text.encode('utf-8'))
            
        # save the result to local database        
        insert_geeks(cursor, js, check = True)
        db.commit()
        print('finish %d' % i)
        # sleep for random time
        time.sleep(random.random())